{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/ritheshkumar95/pytorch-vqvae/tree/master\n",
    "\n",
    "https://arxiv.org/pdf/1606.05908\n",
    "https://arxiv.org/pdf/1906.02691"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions.normal import Normal\n",
    "from torch.distributions.kl import kl_divergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(nn.Module):\n",
    "    def __init__(self, input_dim, dim, z_dim):\n",
    "        super().__init__()\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Conv2d(input_dim, dim, 4, 2, 1),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.Conv2d(dim, dim, 4, 2, 1),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.Conv2d(dim, dim, 5, 1, 0),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.Conv2d(dim, z_dim * 2, 3, 1, 0),\n",
    "            nn.BatchNorm2d(z_dim * 2)\n",
    "        )\n",
    "\n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.ConvTranspose2d(z_dim, dim, 3, 1, 0),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.ConvTranspose2d(dim, dim, 5, 1, 0),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.ConvTranspose2d(dim, dim, 4, 2, 1),\n",
    "            nn.BatchNorm2d(dim),\n",
    "            nn.ReLU(True),\n",
    "            nn.ConvTranspose2d(dim, input_dim, 4, 2, 1),\n",
    "            nn.Tanh()\n",
    "        )\n",
    "\n",
    "        self.apply(weights_init)\n",
    "\n",
    "    def forward(self, x):\n",
    "        mu, logvar = self.encoder(x).chunk(2, dim=1)\n",
    "\n",
    "        q_z_x = Normal(mu, logvar.mul(.5).exp())\n",
    "        p_z = Normal(torch.zeros_like(mu), torch.ones_like(logvar))\n",
    "        kl_div = kl_divergence(q_z_x, p_z).sum(1).mean()\n",
    "\n",
    "        x_tilde = self.decoder(q_z_x.rsample())\n",
    "        return x_tilde, kl_div"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "swinuneter",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
